
import scrapy

headers={
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36'
}

class QuotesV2exSpider(scrapy.Spider):
    name = "quotes_v2ex"
    # start_urls = ["https://www.v2ex.com/recent"]

    async def start(self):
        yield scrapy.Request("https://www.v2ex.com/recent",headers=headers)

    def parse(self, response):
        lines = response.xpath('//div[@class="cell item"]')
        next_url = response.xpath('//td[@class="super normal_page_right button"]/@onclick').extract_first()

        for line in lines:
            detail_url = line.xpath('.//a[@class="topic-link"]/@href').get()
            title = line.xpath('.//a[@class="topic-link"]/text()').get()
            yield scrapy.Request(f'https://www.v2ex.com{detail_url}',headers=headers,callback=self.detail_requests)
            print(title,f'https://www.v2ex.com{detail_url}')


        if next_url:
            yield response.follow(f'https://www.v2ex.com{next_url.split("'")[1]}',headers=headers)


    def detail_requests(self,response):
        lines = response.xpath('//div[starts-with(@id,"r_")]')
        for line in lines:
            author = line.xpath('.//a[@class="dark"]/text()').get()
            answer_block = line.xpath('.//div[@class="reply_content"]')
            has_a = answer_block.xpath('./a/text()').get()
            if not has_a:
                answer = answer_block.xpath('./text()').get()
            else:
                if answer_block.xpath('./text()[1]').get() and has_a+answer_block.xpath('./text()[2]').get():
                    answer = answer_block.xpath('./text()[1]').get()+has_a+answer_block.xpath('./text()[2]').get()
                elif answer_block.xpath('./text()[1]').get():
                    answer = has_a+answer_block.xpath('./text()[1]').get()
                else:
                    answer = has_a
                # print('11111111111111111111',answer_block.xpath('./text()[1]').get())
                # print('222222222222222222222',answer_block.xpath('./text()[2]').get())

            print({
                'author':author,
                'answer':answer,
            })



# //div[starts-with(@id,"r_")]//div[@class="reply_content"]
